<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">

<title>module RDoc::TokenStream - rdoc: Ruby Standard Library Documentation</title>


<script src="../js/navigation.js" defer></script>
<script src="../js/search.js" defer></script>
<script src="../js/search_index.js" defer></script>
<script src="../js/searcher.js" defer></script>
<script src="../js/darkfish.js" defer></script>

<script src="../js/jquery-3.2.0.min.js"></script>

<script src="../js/vue.min.js"></script>
<script src="../js/js.cookie.min.js"></script>

<link href="../css/fonts.css" rel="stylesheet">
<link id='rdoccss' href="../css/rdoc.css" rel="stylesheet">
<link href="../css/carbon17.css" rel="stylesheet">

<script type="text/javascript">
  var rdoc_rel_prefix = "../";
  var index_rel_prefix = "../";
  var darkModeCsseHref = "../css/rdoc-dm.css"
  var defaultModeCssHref = "../css/rdoc.css"
  // var cssDarkmode = Cookies.get('darkmode');
  
  if( Cookies.get("darkmode") == "true") {
	$('#rdoccss').attr("href", darkModeCsseHref);
}

//  https://cssdeck.com/blog/simple-jquery-stylesheet-switcher/

document.write('<style type="text/css">body{display:none}</style>');

</script>


</head>
<body id="top" role="document" class="module">
  <!-- this is class.html -->

  <div id='actionbar' >
    <div class='wrapper mdiv'>
      <ul class='grids g0'></ul>
    </div> 
    <!-- VERSION HEADER for 3.3.0.preview2 NOT FOUND -->
  </div> <!-- end action bar -->

  <div class='wrapper hdiv'>

    


    <nav id='vapp' role="navigation">
    <div id="project-navigation">
      <div id="home-section" role="region" title="Quick navigation" class="nav-section">
  <h2><a href="../index.html" rel="home">Home</a></h2>

  <div id="table-of-contents-navigation"  >
    <a href="../table_of_contents.html#pages">Pages</a>
    <a href="../table_of_contents.html#classes">Classes</a>
    <a href="../table_of_contents.html#methods">Methods</a>
  </div>
</div>

      <div id="search-section" role="search" class="project-section initially-hidden">
  <form action="#" method="get" accept-charset="utf-8">
    <div id="search-field-wrapper">
      <input id="search-field" role="combobox" aria-label="Search"
             aria-autocomplete="list" aria-controls="search-results"
             type="text" name="search" placeholder="Search" spellcheck="false"
             title="Type to search, Up and Down to navigate, Enter to load">
    </div>

    <ul id="search-results" aria-label="Search Results"
        aria-busy="false" aria-expanded="false"
        aria-atomic="false" class="initially-hidden"></ul>
  </form>
</div>

    </div>


    

    <button id='toggleThing' @click="toggleNav()" >Show/hide navigation</button>
    <div :class="isOpen ? 'block' : 'hidden' " id='toggleMe'>
      <div id="class-metadata">
        
        
        
        
        
<!-- Method Quickref -->
<div id="method-list-section" class="nav-section">
  <h3>Methods</h3>

  <ul class="link-list" role="directory">
    <li ><a href="#method-c-to_html">::to_html</a>
    <li ><a href="#method-i-add_token">#add_token</a>
    <li ><a href="#method-i-add_tokens">#add_tokens</a>
    <li ><a href="#method-i-collect_tokens">#collect_tokens</a>
    <li ><a href="#method-i-pop_token">#pop_token</a>
    <li ><a href="#method-i-start_collecting_tokens">#start_collecting_tokens</a>
    <li ><a href="#method-i-token_stream">#token_stream</a>
    <li ><a href="#method-i-tokens_to_s">#tokens_to_s</a>
  </ul>
</div>

      </div>
     </div>
    </nav>


    <div id='extraz'><div class='adzbox-index'  >
      
     </div>         
    </div>

    <main role="main" aria-labelledby="module-RDoc::TokenStream">
    <h1 id="module-RDoc::TokenStream" class="module">
      module RDoc::TokenStream
    </h1>

    <section class="description">
    
<p>A <a href="TokenStream.html"><code>TokenStream</code></a> is a list of tokens, gathered during the parse of some entity (say a method). Entities populate these streams by being registered with the lexer. Any class can collect tokens by including <a href="TokenStream.html"><code>TokenStream</code></a>. From the outside, you use such an object by calling the <a href="TokenStream.html#method-i-start_collecting_tokens"><code>start_collecting_tokens</code></a> method, followed by calls to <a href="TokenStream.html#method-i-add_token"><code>add_token</code></a> and pop_token.</p>

    </section>

      <section id="5Buntitled-5D" class="documentation-section">





                <section id="public-class-5Buntitled-5D-method-details" class="method-section">
                <header>
                <h3>Public Class Methods</h3>
                </header>

                  <div id="method-c-to_html" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">to_html</span><span
                                class="method-args">(token_stream)</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Converts <code>token_stream</code> to HTML wrapping various tokens with <code>&lt;span&gt;</code> elements. Some tokens types are wrapped in spans with the given class names. Other token types are not wrapped in spans.</p>

                              <div class="method-source-code" id="to_html-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 16</span>
<span class="ruby-keyword">def</span> <span class="ruby-keyword">self</span>.<span class="ruby-identifier ruby-title">to_html</span> <span class="ruby-identifier">token_stream</span>
  <span class="ruby-identifier">starting_title</span> = <span class="ruby-keyword">false</span>

  <span class="ruby-identifier">token_stream</span>.<span class="ruby-identifier">map</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">t</span><span class="ruby-operator">|</span>
    <span class="ruby-keyword">next</span> <span class="ruby-keyword">unless</span> <span class="ruby-identifier">t</span>

    <span class="ruby-identifier">style</span> = <span class="ruby-keyword">case</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:kind</span>]
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_const</span>   <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-constant&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_kw</span>      <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-keyword&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_ivar</span>    <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-ivar&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_cvar</span>    <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-identifier&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_gvar</span>    <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-identifier&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-string">&#39;=&#39;</span> <span class="ruby-operator">!=</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:text</span>] <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-value">:on_op</span>
                             <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-operator&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_tlambda</span> <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-operator&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_ident</span>   <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-identifier&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_label</span>   <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-value&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_backref</span>, <span class="ruby-value">:on_dstring</span>
                             <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-node&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_comment</span> <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-comment&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_embdoc</span>  <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-comment&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_regexp</span>  <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-regexp&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_tstring</span> <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-string&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_int</span>, <span class="ruby-value">:on_float</span>,
                 <span class="ruby-value">:on_rational</span>, <span class="ruby-value">:on_imaginary</span>,
                 <span class="ruby-value">:on_heredoc</span>,
                 <span class="ruby-value">:on_symbol</span>, <span class="ruby-value">:on_CHAR</span> <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-value&#39;</span>
            <span class="ruby-keyword">when</span> <span class="ruby-value">:on_heredoc_beg</span>, <span class="ruby-value">:on_heredoc_end</span>
                             <span class="ruby-keyword">then</span> <span class="ruby-string">&#39;ruby-identifier&#39;</span>
            <span class="ruby-keyword">end</span>

    <span class="ruby-identifier">comment_with_nl</span> = <span class="ruby-keyword">false</span>
    <span class="ruby-keyword">if</span> <span class="ruby-value">:on_comment</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:kind</span>] <span class="ruby-keyword">or</span> <span class="ruby-value">:on_embdoc</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:kind</span>] <span class="ruby-keyword">or</span> <span class="ruby-value">:on_heredoc_end</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:kind</span>]
      <span class="ruby-identifier">comment_with_nl</span> = <span class="ruby-keyword">true</span> <span class="ruby-keyword">if</span> <span class="ruby-string">&quot;\n&quot;</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:text</span>][<span class="ruby-value">-1</span>]
      <span class="ruby-identifier">text</span> = <span class="ruby-identifier">t</span>[<span class="ruby-value">:text</span>].<span class="ruby-identifier">rstrip</span>
    <span class="ruby-keyword">else</span>
      <span class="ruby-identifier">text</span> = <span class="ruby-identifier">t</span>[<span class="ruby-value">:text</span>]
    <span class="ruby-keyword">end</span>

    <span class="ruby-keyword">if</span> <span class="ruby-value">:on_ident</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:kind</span>] <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-identifier">starting_title</span>
      <span class="ruby-identifier">starting_title</span> = <span class="ruby-keyword">false</span>
      <span class="ruby-identifier">style</span> = <span class="ruby-string">&#39;ruby-identifier ruby-title&#39;</span>
    <span class="ruby-keyword">end</span>

    <span class="ruby-keyword">if</span> <span class="ruby-value">:on_kw</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:kind</span>] <span class="ruby-keyword">and</span> <span class="ruby-string">&#39;def&#39;</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">t</span>[<span class="ruby-value">:text</span>]
      <span class="ruby-identifier">starting_title</span> = <span class="ruby-keyword">true</span>
    <span class="ruby-keyword">end</span>

    <span class="ruby-identifier">text</span> = <span class="ruby-constant">CGI</span>.<span class="ruby-identifier">escapeHTML</span> <span class="ruby-identifier">text</span>

    <span class="ruby-keyword">if</span> <span class="ruby-identifier">style</span> <span class="ruby-keyword">then</span>
      <span class="ruby-node">&quot;&lt;span class=\&quot;#{style}\&quot;&gt;#{text}&lt;/span&gt;#{&quot;\n&quot; if comment_with_nl}&quot;</span>
    <span class="ruby-keyword">else</span>
      <span class="ruby-identifier">text</span>
    <span class="ruby-keyword">end</span>
  <span class="ruby-keyword">end</span>.<span class="ruby-identifier">join</span>
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                          </section>

                <section id="public-instance-5Buntitled-5D-method-details" class="method-section">
                <header>
                <h3>Public Instance Methods</h3>
                </header>

                  <div id="method-i-add_token" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">add_token</span><span
                                class="method-args">(token)</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Adds one <code>token</code> to the collected tokens</p>

                              <div class="method-source-code" id="add_token-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 84</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">add_token</span>(<span class="ruby-identifier">token</span>)
  <span class="ruby-ivar">@token_stream</span>.<span class="ruby-identifier">push</span>(<span class="ruby-identifier">token</span>)
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                  <div id="method-i-add_tokens" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">add_tokens</span><span
                                class="method-args">(tokens)</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Adds <code>tokens</code> to the collected tokens</p>

                              <div class="method-source-code" id="add_tokens-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 77</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">add_tokens</span>(<span class="ruby-identifier">tokens</span>)
  <span class="ruby-ivar">@token_stream</span>.<span class="ruby-identifier">concat</span>(<span class="ruby-identifier">tokens</span>)
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                  <div id="method-i-collect_tokens" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">collect_tokens</span><span
                                class="method-args">()</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Starts collecting tokens</p>

                              <div class="method-source-code" id="collect_tokens-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 91</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">collect_tokens</span>
  <span class="ruby-ivar">@token_stream</span> = []
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>

                            <div class="aliases">
                              Also aliased as: <a href="TokenStream.html#method-i-start_collecting_tokens">start_collecting_tokens</a>
                            </div>

                          </div>

                  <div id="method-i-pop_token" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">pop_token</span><span
                                class="method-args">()</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Remove the last token from the collected tokens</p>

                              <div class="method-source-code" id="pop_token-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 100</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">pop_token</span>
  <span class="ruby-ivar">@token_stream</span>.<span class="ruby-identifier">pop</span>
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                  <div id="method-i-start_collecting_tokens" class="method-detail method-alias">
                            <div class="method-heading">
                              <span class="method-name">start_collecting_tokens</span><span
                                class="method-args">()</span>
                            </div>

                            <div class="method-description">
                              

                            </div>


                            <div class="aliases">
                              Alias for: <a href="TokenStream.html#method-i-collect_tokens">collect_tokens</a>
                            </div>
                          </div>

                  <div id="method-i-token_stream" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">token_stream</span><span
                                class="method-args">()</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Current token stream</p>

                              <div class="method-source-code" id="token_stream-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 107</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">token_stream</span>
  <span class="ruby-ivar">@token_stream</span>
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                  <div id="method-i-tokens_to_s" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">tokens_to_s</span><span
                                class="method-args">()</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              <p>Returns a string representation of the token stream</p>

                              <div class="method-source-code" id="tokens_to_s-source">
            <pre><span class="ruby-comment"># File rdoc/token_stream.rb, line 114</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">tokens_to_s</span>
  <span class="ruby-identifier">token_stream</span>.<span class="ruby-identifier">compact</span>.<span class="ruby-identifier">map</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">token</span><span class="ruby-operator">|</span> <span class="ruby-identifier">token</span>[<span class="ruby-value">:text</span>] }.<span class="ruby-identifier">join</span> <span class="ruby-string">&#39;&#39;</span>
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                          </section>

              </section>
              </main>



            </div>  <!--  class='wrapper hdiv' -->


<footer id="validator-badges" role="contentinfo">
<p><a href="https://validator.w3.org/check/referer">Validate</a></p>
<p>Generated by <a href="https://ruby.github.io/rdoc/">RDoc</a> 6.4.0.</p>
<p>Based on <a href="https://github.com/ged/darkfish/">Darkfish</a> by <a href="http://deveiate.org">Michael Granger</a>.</p>

  
    <p><p><a href="https://ruby-doc.org">Ruby-doc.org</a> is provided by <a href="https://jamesbritt.com">James Britt</a> and <a href="https://neurogami.com">Neurogami</a>.</p><p><a href="https://jamesbritt.bandcamp.com/">Maximum R+D</a>.  </p>
</p>
  
  </footer>

<script type="text/javascript">


  let ads  = $("#carbonads-container").children().detach();


  function swapMode() {
    var cookieName = 'darkmode';
    var cssDarkmode = Cookies.get(cookieName);
    console.log("***** swapMode! " + cssDarkmode + " *****");


    if (cssDarkmode == "true") {
      console.log("We have dark mode, set the css to light ...");
      $('#rdoccss').attr("href", defaultModeCssHref);
      $('#cssSelect').text("Dark mode");
      cssDarkmode = "false";
      console.log("swapMode! Now set cookie to " + cssDarkmode);
      Cookies.set(cookieName, cssDarkmode);

    } else {
      console.log("We not have dark mode, set the css to dark ...");
      $('#rdoccss').attr("href", darkModeCsseHref);
      $('#cssSelect').text("Light mode");
      cssDarkmode = "true";
      console.log("swapMode! Now set cookie to " + cssDarkmode);
      Cookies.set(cookieName, cssDarkmode);

    }

    console.log("  --------------- ");
  }


const vueCssApp = new Vue({
el: '#menubar',
data: {
isDark: false
},
methods: {
toggleClass: function(event){
this.isDark = !this.isDark;
}
}
})

const vueApp = new Vue({
el: '#vapp',
data: { 
isOpen: true
},

mounted() {
this.handleResize();
this.manage_mob_classes();
window.addEventListener('resize', this.handleResize)
//this.isOpen !=  (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent));
},
destroyed() {
window.removeEventListener('resize', this.handleResize)
},
created() {
//manage_mob_classes();
},

methods : {
isMobile() {
  return (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent));
},

  handleResize() {
    if (!this.isMobile()) {
      this.isOpen = window.innerWidth > 800;
    }
  },

  manage_mob_classes() {
    if (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
      $("nav").addClass("mob_nav");
      $("main").addClass("mob_main");
      $("#extraz").addClass("mob_extraz");
      $("#carbonads-container").addClass("mob_carbonads-container");
      this.isOpen  = false;
    } else {
      $("nav").removeClass("mob_nav") 
        $("main").removeClass("mob_main");
      $("#extraz").removeClass("mob_extraz");
      $("#carbonads-container").removeClass("mob_carbonads-container");
      this.isOpen  = true;
    }
  },

  toggleNav() {
    this.isOpen =! this.isOpen ;
    // alert("Toggle nav!");
    console.log("toggleNav() click: " + this.isOpen );
  }
}
})

$("#carbonads-container").append(ads);


$(function() {

    var darkmode = Cookies.get("darkmode");
    console.log("Document ready: " + darkmode);

    if ( darkmode  == "true" ) {
      $('#cssSelect').text("Light mode");
    } else {
      $('#cssSelect').text("Dark mode");
     }

    $('body').css('display','block');
    });

</script>

    
  </body> 
</html>

